<!DOCTYPE html>
<html lang="zh-CN">
<head>
  <meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=2">
<meta name="theme-color" content="#222">
<meta name="generator" content="Hexo 3.9.0">
  <link rel="apple-touch-icon" sizes="180x180" href="/images/apple-touch-icon-next.png">
  <link rel="icon" type="image/png" sizes="32x32" href="/images/favicon-32x32-next.png">
  <link rel="icon" type="image/png" sizes="16x16" href="/images/favicon-16x16-next.png">
  <link rel="mask-icon" href="/images/logo.svg" color="#222">

<link rel="stylesheet" href="/css/main.css">


<link rel="stylesheet" href="//cdn.jsdelivr.net/npm/@fortawesome/fontawesome-free@5/css/all.min.css">

<script id="hexo-configurations">
    var NexT = window.NexT || {};
    var CONFIG = {"hostname":"ywg.imfast.io","root":"/","scheme":"Muse","version":"7.8.0","exturl":false,"sidebar":{"position":"right","Muse | Mist":320,"display":"post","padding":18,"offset":12,"onmobile":true},"copycode":{"enable":true,"show_result":true,"style":null},"back2top":{"enable":true,"sidebar":false,"scrollpercent":false},"bookmark":{"enable":false,"color":"#222","save":"auto"},"fancybox":false,"mediumzoom":false,"lazyload":false,"pangu":false,"comments":{"style":"tabs","active":null,"storage":true,"lazyload":false,"nav":null},"algolia":{"hits":{"per_page":10},"labels":{"input_placeholder":"Search for Posts","hits_empty":"We didn't find any results for the search: ${query}","hits_stats":"${hits} results found in ${time} ms"}},"localsearch":{"enable":false,"trigger":"auto","top_n_per_article":1,"unescape":false,"preload":false},"motion":{"enable":false,"async":false,"transition":{"post_block":"fadeIn","post_header":"slideDownIn","post_body":"slideDownIn","coll_header":"slideLeftIn","sidebar":"slideUpIn"}}};
  </script>

  <meta name="description" content="HTML 用于展示数据，而 XML 用于结构化、传输和存储数据 XML 文档必须有一个根元素；     嵌套的 HTML 元素,结构化的一种文档结构 ——&amp;gt; W3C 针对其指定了一个标准化的模型DOM  json数据获取方式:     ajax 请求接口     切换手机移动h5端     app抓包获取     等等json模块：⽤于字符串和 python 数据类型间进⾏转换     j">
<meta name="keywords" content="爬虫,python">
<meta property="og:type" content="article">
<meta property="og:title" content="spider">
<meta property="og:url" content="https://ywg.imfast.io/2018/11/26/spider/index.html">
<meta property="og:site_name" content="中·庸">
<meta property="og:description" content="HTML 用于展示数据，而 XML 用于结构化、传输和存储数据 XML 文档必须有一个根元素；     嵌套的 HTML 元素,结构化的一种文档结构 ——&amp;gt; W3C 针对其指定了一个标准化的模型DOM  json数据获取方式:     ajax 请求接口     切换手机移动h5端     app抓包获取     等等json模块：⽤于字符串和 python 数据类型间进⾏转换     j">
<meta property="og:locale" content="zh-CN">
<meta property="og:updated_time" content="2020-04-21T11:30:50.568Z">
<meta name="twitter:card" content="summary">
<meta name="twitter:title" content="spider">
<meta name="twitter:description" content="HTML 用于展示数据，而 XML 用于结构化、传输和存储数据 XML 文档必须有一个根元素；     嵌套的 HTML 元素,结构化的一种文档结构 ——&amp;gt; W3C 针对其指定了一个标准化的模型DOM  json数据获取方式:     ajax 请求接口     切换手机移动h5端     app抓包获取     等等json模块：⽤于字符串和 python 数据类型间进⾏转换     j">

<link rel="canonical" href="https://ywg.imfast.io/2018/11/26/spider/">


<script id="page-configurations">
  // https://hexo.io/docs/variables.html
  CONFIG.page = {
    sidebar: "",
    isHome : false,
    isPost : true,
    lang   : 'zh-CN'
  };
</script>

  <title>spider | 中·庸</title>
  


  <script data-pjax>
    var _hmt = _hmt || [];
    (function() {
      var hm = document.createElement("script");
      hm.src = "https://hm.baidu.com/hm.js?5b0bac5c2f7738d023171c65307bcc43";
      var s = document.getElementsByTagName("script")[0];
      s.parentNode.insertBefore(hm, s);
    })();
  </script>




  <noscript>
  <style>
  .use-motion .brand,
  .use-motion .menu-item,
  .sidebar-inner,
  .use-motion .post-block,
  .use-motion .pagination,
  .use-motion .comments,
  .use-motion .post-header,
  .use-motion .post-body,
  .use-motion .collection-header { opacity: initial; }

  .use-motion .site-title,
  .use-motion .site-subtitle {
    opacity: initial;
    top: initial;
  }

  .use-motion .logo-line-before i { left: initial; }
  .use-motion .logo-line-after i { right: initial; }
  </style>
</noscript>

</head>

<body itemscope itemtype="http://schema.org/WebPage">
  <div class="container">
    <div class="headband"></div>

    <header class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner"><div class="site-brand-container">
  <div class="site-nav-toggle">
    <div class="toggle" aria-label="切换导航栏">
      <span class="toggle-line toggle-line-first"></span>
      <span class="toggle-line toggle-line-middle"></span>
      <span class="toggle-line toggle-line-last"></span>
    </div>
  </div>

  <div class="site-meta">

    <a href="/" class="brand" rel="start">
      <span class="logo-line-before"><i></i></span>
      <h1 class="site-title">中·庸</h1>
      <span class="logo-line-after"><i></i></span>
    </a>
      <p class="site-subtitle" itemprop="description">斯人若彩虹，遇上方知有</p>
  </div>

  <div class="site-nav-right">
    <div class="toggle popup-trigger">
    </div>
  </div>
</div>




<nav class="site-nav">
  <ul id="menu" class="main-menu menu">
        <li class="menu-item menu-item-home">

    <a href="/" rel="section"><i class="home fa-fw"></i>首页</a>

  </li>
        <li class="menu-item menu-item-tags">

    <a href="/tags/" rel="section"><i class="tags fa-fw"></i>标签<span class="badge">9</span></a>

  </li>
        <li class="menu-item menu-item-categories">

    <a href="/categories/" rel="section"><i class="th fa-fw"></i>分类<span class="badge">9</span></a>

  </li>
        <li class="menu-item menu-item-archives">

    <a href="/archives/" rel="section"><i class="archive fa-fw"></i>归档<span class="badge">10</span></a>

  </li>
        <li class="menu-item menu-item-about">

    <a href="/about/" rel="section"><i class="user fa-fw"></i>关于</a>

  </li>
  </ul>
</nav>




</div>
    </header>

    
  <div class="back-to-top">
    <i class="fa fa-arrow-up"></i>
    <span>0%</span>
  </div>


    <main class="main">
      <div class="main-inner">
        <div class="content-wrap">
          

          <div class="content post posts-expand">
            

    
  
  
  <article itemscope itemtype="http://schema.org/Article" class="post-block" lang="zh-CN">
    <link itemprop="mainEntityOfPage" href="https://ywg.imfast.io/2018/11/26/spider/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="image" content="/images/avatar.gif">
      <meta itemprop="name" content="zhiheng">
      <meta itemprop="description" content="programming and life">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="中·庸">
    </span>
      <header class="post-header">
        <h1 class="post-title" itemprop="name headline">
          spider
        </h1>

        <div class="post-meta">
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="far fa-calendar"></i>
              </span>
              <span class="post-meta-item-text">发表于</span>

              <time title="创建时间：2018-11-26 17:08:58" itemprop="dateCreated datePublished" datetime="2018-11-26T17:08:58+08:00">2018-11-26</time>
            </span>
              <span class="post-meta-item">
                <span class="post-meta-item-icon">
                  <i class="far fa-calendar-check"></i>
                </span>
                <span class="post-meta-item-text">更新于</span>
                <time title="修改时间：2020-04-21 19:30:50" itemprop="dateModified" datetime="2020-04-21T19:30:50+08:00">2020-04-21</time>
              </span>
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="far fa-folder"></i>
              </span>
              <span class="post-meta-item-text">分类于</span>
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/爬虫/" itemprop="url" rel="index"><span itemprop="name">爬虫</span></a>
                </span>
            </span>

          
            <span class="post-meta-item" title="阅读次数" id="busuanzi_container_page_pv" style="display: none;">
              <span class="post-meta-item-icon">
                <i class="fa fa-eye"></i>
              </span>
              <span class="post-meta-item-text">阅读次数：</span>
              <span id="busuanzi_value_page_pv"></span>
            </span>

        </div>
      </header>

    
    
    
    <div class="post-body" itemprop="articleBody">

      
        <p> HTML 用于展示数据，而 XML 用于结构化、传输和存储数据 XML 文档必须有一个根元素；<br>     嵌套的 HTML 元素,结构化的一种文档结构 ——&gt; W3C 针对其指定了一个标准化的模型DOM </p>
<h3 id="json数据"><a href="#json数据" class="headerlink" title="json数据"></a>json数据</h3><p>获取方式:<br>     ajax 请求接口<br>     切换手机移动h5端<br>     app抓包获取<br>     等等<br>json模块：⽤于字符串和 python 数据类型间进⾏转换<br>     js：JSON.stringify   JSON.parse</p>
<blockquote>
<p>只有基本数据类型才能转换成JSON格式字符串</p>
</blockquote>
<h3 id="python封装http库：-请求的是源代码"><a href="#python封装http库：-请求的是源代码" class="headerlink" title="python封装http库：# 请求的是源代码"></a>python封装http库：# 请求的是源代码</h3><pre><code>标准urllib库
第三方库request</code></pre><ol>
<li><p>urllib库：</p>
<ol>
<li><p>request模块：<br>  urlopen()函数</p>
<pre><code>urllib的params需urlencode转换类型
data参数：请求方式变为post  get方式一般需拼接query_string
     urllib.parse.urlencode(form_data).encode() // POST data should be bytes, an iterable of bytes, or a file object. It cannot be of type str.
超时处理：设置timeout
     read() 返回二进制代码 data=urllib.request.urlopen(url).read().decode(&quot;UTF-8&quot;,&quot;ignore&quot;)</code></pre><p>  urlretrieve(图片的源地址,保存文件的文件名)  ——打开并保存url内容 重新请求url响应 就实现了请求图片、新建文件、写入文件三个步骤的功能。<br>  Request类 :构造request请求对象  (url, data=None, headers={},origin_req_host=None, unverifiable=False,method=None)<br>  Handler类<br>response:<br>  read()  读取字节类型<br>  geturl()    获取请求url<br>  getheaders()<br>  getcode()<br>  readlines()     </p>
</li>
<li><p>error模块：<br>  URLError类（作用于request模块的异常）<br>  HTTPError类(子类)</p>
<pre><code>三个属性code reason headers

解析异常：AttributeError：
标签为None</code></pre></li>
<li><p>parse模块：（处理url）<br>  解析urlparse() (根据url的6个部分解析)<br>  构造urlunparse() params接受长度为6的可迭代对象</p>
<pre><code>urlsplit()</code></pre><p>  合并urljoin()  # url自动修正拼接完整  </p>
<pre><code>- 拼接完整url
     if item[&apos;href&apos;] is not None:
          item[&apos;href&apos;] = urllib.parse.urljoin(response.url, item[&apos;href&apos;])</code></pre><p>  url编码转换：url中若出现 $ 空格 中文等，就要对其进行编码</p>
<pre><code>1.字符串：urllib.parse.quote(str)  url编码
   unquote() url解码  # requests.utils.unquote(str,encoding=&apos;utf-8&apos;, errors=&apos;replace&apos;)
2.字典：  urllib.parse.urlencode(dict) 转化为url参数</code></pre></li>
</ol>
</li>
<li><p>requests库 - 添加/修改请求头</p>
<ul>
<li><p>GET（url,params,headers[cookies=],timeout）   使用 cookies 参数，首先需要将 Cookie 值处理为字典的形式：<br>params:dict =&gt; query-data<br>headers = {</p>
<pre><code>cookie,</code></pre><p>}<br>当把get函数的stream参数设置成True时，它不会立即开始下载，当你使用iter_content或iter_lines遍历内容或访问内容属性时才开始下载。需要注意一点：文件没有下载之前，它也需要保持连接。<br>iter_content：一块一块的遍历要下载的内容<br>iter_lines：一行一行的遍历要下载的内容<br>使用上面两个函数下载大文件可以防止占用过多的内存，因为每次只下载小部分数据。</p>
<pre><code>示例代码：
1 r = requests.get(url_file, stream=True)
2 f = open(&quot;file_path&quot;, &quot;wb&quot;)
3 for chunk in r.iter_content(chunk_size=512):
4     if chunk:
5         f.write(chunk)</code></pre></li>
<li><p>post(url,data{},headers,cookies,files={‘avator’:open(r’1.jpg’,’wb’)})   </p>
<ol>
<li><p>处理登录：声明一个session会话，自动处理cookies</p>
<pre><code>操作session获取数据： session.cookies.save()
res.cookies.get_dict()</code></pre></li>
<li><p>证书验证（https协议）</p>
<ol>
<li>设置认证verify=False</li>
<li>指定cert=（path）</li>
</ol>
</li>
<li><p>代理设置<br>  (proxies=..)</p>
</li>
<li><p>requests异常处理：<br>  from requests.exceptions import RequestException</p>
</li>
<li><p>auth 内网认证<br>  auth = (‘user’,’pass’)</p>
</li>
</ol>
<p>res方法：</p>
<ul>
<li>获取服务器的原始套接字响应:r.raw  请求：(,stream=True)<br>   边下载边存硬盘:for chunk in r.iter_content(chunk_size=1024):</li>
</ul>
<p>r.json() 获取json数据  相当于content = response.content  results = json.loads(content)</p>
<p>获取cookies字典：response.cookies.get_dict()</p>
<p><strong>response属性</strong>（6）响应头响应体相关:  r.request.headers</p>
</li>
<li><p>r.status_code</p>
<p>   http请求的返回状态，200表示连接成功，404表示连接失败没有对应资源<br>   301/302 永久/临时重定向<br>   403 没有权限访问 400：客户端请求语法错误 404：请求的资源没有找到<br>   500 服务器错误 503 服务器正在维护</p>
</li>
<li><p>r.text</p>
<p>   http响应<strong>解码</strong>后的str形式，根据r.encoding</p>
</li>
<li><p>r.encoding  从HTTP header中猜测的响应内容编码方式</p>
</li>
<li><p>r.apparent_encoding  从内容分析出的响应内容的编码方式（备选编码方式）</p>
<pre><code># 乱码解决方案   设置编码: res.encoding=res.apparent_encoding</code></pre></li>
<li><p>r.content .decode()</p>
<p>   HTTP响应内容的编码二进制形式bytes类型</p>
</li>
<li><p>r.headers</p>
<p>   http响应内容的头部内容</p>
</li>
</ul>
</li>
</ol>
<blockquote>
<p>利用 XPath css<br>选择器来提取定位某个节点，然后再调用相应方法获取它的正文内容或者属性</p>
</blockquote>
<p>3个解析库：bs，lxml，pyquery</p>
<h3 id="bs4（网页解析库，支持多种解析器’html-parser’-‘lxml’）"><a href="#bs4（网页解析库，支持多种解析器’html-parser’-‘lxml’）" class="headerlink" title="bs4（网页解析库，支持多种解析器’html.parser’ ‘lxml’）"></a>bs4（网页解析库，支持多种解析器’html.parser’ ‘lxml’）</h3><pre><code> 初始化一个soup对象，指定解析库
      &gt; bs自动将输入文档转 Unicode 编码，输出文档转换为 UTF-8 编码 你不需考虑编码方式（bs将HTML文档转为DOM树）

      soup = BeautifulSoup(open(&apos;index.html&apos;),&apos;html.parser&apos;) #打开本地 HTML ⽂件的⽅式来创建对象


 方法：
 1. soup.find_all(tag, attributes, recursive, text, limit限制个数, keywords)   // &apos;class_&apos;防止冲突
      参数值可以是字符串 或正则re.compile(&apos;&apos;)
      返回list类型的html源码,find方法返回bs.BeautifulSoup对象

    在 BeautifulSoup 中每个元素都有一个 get_text() 获取文本   取属性：get（属性名）
    - 取标签: soup.p标签：默认只取第一个 soup.p.string取文本
    - 获取属性：
    soup.attrs # 返回标签属性dict  
    element.text提取text文本内容
    soup.a[&apos;href&apos;]   

    获取属性们：
      ···
      nameList = bs.find_all(text=&apos;the prince&apos;,attrs=｛’id＇：’list&apos;｝)
      [&apos;the prince&apos;, &apos;the prince&apos;, &apos;the prince&apos;, &apos;the prince&apos;, &apos;the prince&apos;, &apos;the prince&apos;, &apos;the prince&apos;]
      ···

      bs节点会包括&apos;\n&apos; &apos; &apos;,一般不用  replace.(&apos;\xa0&apos;*8,&apos;\n\n&apos;)替换8个空格为回车分段(HTML中为&amp;nbsp)
    - 获取直接子节点: div.contents                
      处理同辈节点(siblings)和父辈节点(parents):
           next_sibling　next_siblings 返回后面的兄弟节点　　previous_siblings　　返回前面的兄弟节点

 2. soup.select(&apos;css选择器&apos;，limit=1)     //select选择器返回list； css选择器更简洁

      - css选择器：标签名与类名、id 名组合   
      h1::text  ::attr(属性)获取标签属性 伪类选择器  [xpath @属性]

        li:nth-child(3)选取第三个li元素
        li:nth-child(2n) 第偶数个
        a[title]选取有title属性的a元素
        .container.wrapper 表示同一标签class
        ul p 选取所有子元素，包括嵌套不相关
        ul + p 选取ul后面第一个p元素(下一个)   ~所有兄弟
        div#container &gt; ul 第一个子元素

* soup.prettify() 以标准缩进输出代码</code></pre><p>  *** lxml解析html代码和文件<br>        parse_text: etree.HTML(text)  #直接读取字符串<br>        parse_file：etree.parse()     #文件读取<br>        # 按字符串序列化 HTML文档<br>        result = etree.tostring(html)</p>
<pre><code> - Xpath：在xml文档中选取元素属性的路径表达式

      下一级xpath加  .//选中文档所有子孙标签  ./    
      //*[@class=&apos;&apos;]    
      / 选中直接子节点  //开头遍历所有  //选中文档所有子孙标签  
      */@属性：获取元素对应属性值  tag（）获取标签名   text（）获取文本    

      下标a[1]  li[last()-1] 可以找到倒二个元素  /a/..定位父节点

      模糊查询/[contains()]   
      a[text()=&apos;下一页&apos;] ：class名称相同时用

四个基本方法：
 .extract（） 序列化该节点为 Unicode 字符串并返回 list
 .css(): 返回该表达式所对应的所有节点的 selector list，语法同 BeautifulSoup4
 .re(): 返回 Unicode 字符串 list 正则只有\d才能提取数字</code></pre><h3 id="pyquery选择器-from-pyquery-import-PyQuery-as-pq"><a href="#pyquery选择器-from-pyquery-import-PyQuery-as-pq" class="headerlink" title="pyquery选择器  from pyquery import PyQuery as pq"></a>pyquery选择器  from pyquery import PyQuery as pq</h3><pre><code>方法：
find()查找子孙节点  children（）查找所有子节点
多个节点.items()方法 返回一个generator

html() attr() 返回第一个节点
text() 返回所有text 中间空格分开，str类型</code></pre><h2 id="js动态渲染问题："><a href="#js动态渲染问题：" class="headerlink" title="js动态渲染问题："></a>js动态渲染问题：</h2><h3 id="模拟Ajax异步数据爬取-（返回json类型）"><a href="#模拟Ajax异步数据爬取-（返回json类型）" class="headerlink" title="模拟Ajax异步数据爬取 （返回json类型）"></a>模拟Ajax异步数据爬取 （返回json类型）</h3><pre><code>Type：xhr</code></pre><p>cookies池，代理池，UA（from fake_useragent import UserAgent）</p>
<p>协程的优势：<br>     最大的优势就是协程极高的执行效率。因为子程序切换不是线程切换，而是由程序自身控制，因此，没有线程切换的开销，和多线程比，线程数量越多，协程的性能优势就越明显。<br>     第二大优势就是不需要多线程的锁机制，因为只有一个线程，也不存在同时写变量冲突，在协程中控制共享资源不加锁，只需要判断状态就好了，所以执行效率比多线程高很多</p>
<h3 id="实战经验："><a href="#实战经验：" class="headerlink" title="实战经验："></a>实战经验：</h3><pre><code>简单数据，直接用正则提取   
# def write_to_file(content) :
     with open (’result.txt ’,’a ’, encoding=’ utf-8&apos;) as f:   # 写入文件有中文，指定编码格式
          f.write(json.dumps(content ，ensure_ascii=False) +’\n&apos;)   ## json.dumps() 序列化时默认使⽤的 ascii 编码

# json.dump(item,open(&apos;sfda.json&apos;,&apos;w&apos;),ensure_ascii=False)  </code></pre><h1 id="代码片段"><a href="#代码片段" class="headerlink" title="代码片段"></a>代码片段</h1><ul>
<li><p>处理src:<br>item[‘img_list’] = [requests.utils.unquote(i).split(‘src=’)[-1] for i in item[‘img_list’]]</p>
</li>
<li><h1 id="去掉列表中的空字符串和字符串的空白"><a href="#去掉列表中的空字符串和字符串的空白" class="headerlink" title="去掉列表中的空字符串和字符串的空白"></a>去掉列表中的空字符串和字符串的空白</h1><p>content = [i.strip() for i in content if len(i.strip())&gt;0] </p>
</li>
<li><p>scrapy带cookies请求,转换cookies为dict<br>def start_requests(self):</p>
<pre><code>cookies = &quot;anonymid=jqpacc0i4eh2re; depovince=ZGQT; _r01_=1; JSESSIONID=abcM4DE98_8lS_2hu7hLw;&quot;
cookies = {i.split(&apos;=&apos;)[0]:i.split(&apos;=&apos;)[1] for i in cookies.split(&apos;; &apos;)}</code></pre></li>
</ul>
<h3 id="selemium-浏览器自动化"><a href="#selemium-浏览器自动化" class="headerlink" title="selemium 浏览器自动化"></a>selemium 浏览器自动化</h3><pre><code>   from selenium import webdriver
   driver = webdriver.Chrome()
   drive.get(&apos;&apos;)   //打开特定网页
   drive.page_source  //打开网页源码
获取源代码page_source
     获取节点属性等信息
查找节点：（通过XPath/CSS选择器）
     单个节点/多个节点find_element() 返回的是WebElement类型
      * 查找单个节点
          <figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">from</span> selenium <span class="keyword">import</span> webdriver</span><br><span class="line">browser = webdriver.Chrome()</span><br><span class="line">browser.get(<span class="string">'https://www.taobao.com'</span>)</span><br><span class="line">input_ first = browser.find_element_by_id(<span class="string">'q'</span>)</span><br><span class="line">input_second = browser.find_element_by css_selector   (<span class="string">'#q’)</span></span><br><span class="line"><span class="string">input_third = browser_find_element_by_xpath('</span>//*[@id ＝ 飞”］’）</span><br><span class="line">print(input_first, iput_second, input_third)</span><br><span class="line">browser.close()</span><br></pre></td></tr></table></figure>

          另一种参数的灵活方式：
          `find_element(By.ID, id ）`

          * 多个节点
          `find_elements(By.CSS_SELECTOR,&apos;.service-bd li&apos;) `

节点交互：
     动作链ActionChains

     输入文字时用 `send_keys()`方法，清空文字时用` clear()`方法，点击按钮时用 `click()`方法


(万能方法)模拟运行js：execute_script ()

切换框架：switch_to.frame()

加载等待：
     隐式等待browser.implicitly_ wait(10)
显式等待：</code></pre>
    </div>

    
    
    

      <footer class="post-footer">
          <div class="post-tags">
              <a href="/tags/爬虫/" rel="tag"># 爬虫</a>
              <a href="/tags/python/" rel="tag"># python</a>
          </div>

        


        
    <div class="post-nav">
      <div class="post-nav-item">
    <a href="/2018/11/26/你好-Hexo/" rel="prev" title="hexo使用">
      <i class="fa fa-chevron-left"></i> hexo使用
    </a></div>
      <div class="post-nav-item">
    <a href="/2018/11/26/js/" rel="next" title="javascript笔记">
      javascript笔记 <i class="fa fa-chevron-right"></i>
    </a></div>
    </div>
      </footer>
    
  </article>
  
  
  



          </div>
          
    <div class="comments" id="gitalk-container"></div>

<script>
  window.addEventListener('tabs:register', () => {
    let { activeClass } = CONFIG.comments;
    if (CONFIG.comments.storage) {
      activeClass = localStorage.getItem('comments_active') || activeClass;
    }
    if (activeClass) {
      let activeTab = document.querySelector(`a[href="#comment-${activeClass}"]`);
      if (activeTab) {
        activeTab.click();
      }
    }
  });
  if (CONFIG.comments.storage) {
    window.addEventListener('tabs:click', event => {
      if (!event.target.matches('.tabs-comment .tab-content .tab-pane')) return;
      let commentClass = event.target.classList[1];
      localStorage.setItem('comments_active', commentClass);
    });
  }
</script>

        </div>
          
  
  <div class="toggle sidebar-toggle">
    <span class="toggle-line toggle-line-first"></span>
    <span class="toggle-line toggle-line-middle"></span>
    <span class="toggle-line toggle-line-last"></span>
  </div>

  <aside class="sidebar">
    <div class="sidebar-inner">

      <ul class="sidebar-nav motion-element">
        <li class="sidebar-nav-toc">
          文章目录
        </li>
        <li class="sidebar-nav-overview">
          站点概览
        </li>
      </ul>

      <!--noindex-->
      <div class="post-toc-wrap sidebar-panel">
          <div class="post-toc motion-element"><ol class="nav"><li class="nav-item nav-level-3"><a class="nav-link" href="#json数据"><span class="nav-number">1.</span> <span class="nav-text">json数据</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#python封装http库：-请求的是源代码"><span class="nav-number">2.</span> <span class="nav-text">python封装http库：# 请求的是源代码</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#bs4（网页解析库，支持多种解析器’html-parser’-‘lxml’）"><span class="nav-number">3.</span> <span class="nav-text">bs4（网页解析库，支持多种解析器’html.parser’ ‘lxml’）</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#pyquery选择器-from-pyquery-import-PyQuery-as-pq"><span class="nav-number">4.</span> <span class="nav-text">pyquery选择器  from pyquery import PyQuery as pq</span></a></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#js动态渲染问题："><span class="nav-number"></span> <span class="nav-text">js动态渲染问题：</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#模拟Ajax异步数据爬取-（返回json类型）"><span class="nav-number">1.</span> <span class="nav-text">模拟Ajax异步数据爬取 （返回json类型）</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#实战经验："><span class="nav-number">2.</span> <span class="nav-text">实战经验：</span></a></li></ol></li></ol></li><li class="nav-item nav-level-1"><a class="nav-link" href="#代码片段"><span class="nav-number"></span> <span class="nav-text">代码片段</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#去掉列表中的空字符串和字符串的空白"><span class="nav-number"></span> <span class="nav-text">去掉列表中的空字符串和字符串的空白</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#selemium-浏览器自动化"><span class="nav-number">1.</span> <span class="nav-text">selemium 浏览器自动化</span></a></li></ol></div>
      </div>
      <!--/noindex-->

      <div class="site-overview-wrap sidebar-panel">
        <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
  <p class="site-author-name" itemprop="name">zhiheng</p>
  <div class="site-description" itemprop="description">programming and life</div>
</div>
<div class="site-state-wrap motion-element">
  <nav class="site-state">
      <div class="site-state-item site-state-posts">
          <a href="/archives/">
        
          <span class="site-state-item-count">10</span>
          <span class="site-state-item-name">日志</span>
        </a>
      </div>
      <div class="site-state-item site-state-categories">
            <a href="/categories/">
          
        <span class="site-state-item-count">9</span>
        <span class="site-state-item-name">分类</span></a>
      </div>
      <div class="site-state-item site-state-tags">
            <a href="/tags/">
          
        <span class="site-state-item-count">9</span>
        <span class="site-state-item-name">标签</span></a>
      </div>
  </nav>
</div>
  <div class="links-of-author motion-element">
      <span class="links-of-author-item">
        <a href="https://github.com/yuwengeng" title="GitHub → https://github.com/yuwengeng" rel="noopener" target="_blank"><i class="github fa-fw"></i>GitHub</a>
      </span>
      <span class="links-of-author-item">
        <a href="https://www.cnblogs.com/yuwengeng/" title="博客园 → https://www.cnblogs.com/yuwengeng/" rel="noopener" target="_blank"><i class="globe fa-fw"></i>博客园</a>
      </span>
      <span class="links-of-author-item">
        <a href="mailto:yuwenzhiheng95@gmail.com" title="E-Mail → mailto:yuwenzhiheng95@gmail.com" rel="noopener" target="_blank"><i class="envelope fa-fw"></i>E-Mail</a>
      </span>
      <span class="links-of-author-item">
        <a href="https://weibo.com/yourname" title="Weibo → https://weibo.com/yourname" rel="noopener" target="_blank"><i class="weibo fa-fw"></i>Weibo</a>
      </span>
      <span class="links-of-author-item">
        <a href="https://twitter.com/yourname" title="Twitter → https://twitter.com/yourname" rel="noopener" target="_blank"><i class="twitter fa-fw"></i>Twitter</a>
      </span>
      <span class="links-of-author-item">
        <a href="https://instagram.com/yourname" title="Instagram → https://instagram.com/yourname" rel="noopener" target="_blank"><i class="instagram fa-fw"></i>Instagram</a>
      </span>
  </div>



      </div>

    </div>
  </aside>
  <div id="sidebar-dimmer"></div>


      </div>
    </main>

    <footer class="footer">
      <div class="footer-inner">
        

        

<div class="copyright">
  
  &copy; 
  <span itemprop="copyrightYear">2020</span>
  <span class="with-love">
    <i class="heart"></i>
  </span>
  <span class="author" itemprop="copyrightHolder">zhiheng</span>
</div>
  <div class="powered-by">由 <a href="https://hexo.io" class="theme-link" rel="noopener" target="_blank">Hexo</a> & <a href="https://muse.theme-next.org" class="theme-link" rel="noopener" target="_blank">NexT.Muse</a> 强力驱动
  </div>

        
<div class="busuanzi-count">
  <script data-pjax async src="https://busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script>
    <span class="post-meta-item" id="busuanzi_container_site_uv" style="display: none;">
      <span class="post-meta-item-icon">
        <i class="fa fa-user"></i>
      </span>
      <span class="site-uv" title="总访客量">
        <span id="busuanzi_value_site_uv"></span>
      </span>
    </span>
    <span class="post-meta-divider">|</span>
    <span class="post-meta-item" id="busuanzi_container_site_pv" style="display: none;">
      <span class="post-meta-item-icon">
        <i class="fa fa-eye"></i>
      </span>
      <span class="site-pv" title="总访问量">
        <span id="busuanzi_value_site_pv"></span>
      </span>
    </span>
</div>








      </div>
    </footer>
  </div>

  
  <script src="//cdn.jsdelivr.net/npm/animejs@3.1.0/lib/anime.min.js"></script>
  <script src="//cdn.jsdelivr.net/gh/theme-next/theme-next-pjax@0/pjax.min.js"></script>
<script src="/js/utils.js"></script>
<script src="/js/schemes/muse.js"></script>
<script src="/js/next-boot.js"></script>
  <script>
var pjax = new Pjax({
  selectors: [
    'head title',
    '#page-configurations',
    '.content-wrap',
    '.post-toc-wrap',
    '.languages',
    '#pjax'
  ],
  switches: {
    '.post-toc-wrap': Pjax.switches.innerHTML
  },
  analytics: false,
  cacheBust: false,
  scrollTo : !CONFIG.bookmark.enable
});

window.addEventListener('pjax:success', () => {
  document.querySelectorAll('script[data-pjax], script#page-configurations, #pjax script').forEach(element => {
    var code = element.text || element.textContent || element.innerHTML || '';
    var parent = element.parentNode;
    parent.removeChild(element);
    var script = document.createElement('script');
    if (element.id) {
      script.id = element.id;
    }
    if (element.className) {
      script.className = element.className;
    }
    if (element.type) {
      script.type = element.type;
    }
    if (element.src) {
      script.src = element.src;
      // Force synchronous loading of peripheral JS.
      script.async = false;
    }
    if (element.dataset.pjax !== undefined) {
      script.dataset.pjax = '';
    }
    if (code !== '') {
      script.appendChild(document.createTextNode(code));
    }
    parent.appendChild(script);
  });
  NexT.boot.refresh();
  // Define Motion Sequence & Bootstrap Motion.
  if (CONFIG.motion.enable) {
    NexT.motion.integrator
      .init()
      .add(NexT.motion.middleWares.subMenu)
      .add(NexT.motion.middleWares.postList)
      .bootstrap();
  }
  NexT.utils.updateSidebarPosition();
});
</script>




  















    <div id="pjax">
  

  

<link rel="stylesheet" href="//cdn.jsdelivr.net/npm/gitalk@1/dist/gitalk.min.css">

<script>
NexT.utils.loadComments(document.querySelector('#gitalk-container'), () => {
  NexT.utils.getScript('//cdn.jsdelivr.net/npm/gitalk@1/dist/gitalk.min.js', () => {
    var gitalk = new Gitalk({
      clientID    : '24ef63b5f7c6e9d3a198',
      clientSecret: '5cceec5609a0910c5f361eaaefdbe57ae7c5d359',
      repo        : 'yuwengeng.github.io',
      owner       : 'yuwengeng',
      admin       : ['yuwengeng'],
      id          : '2c3b72b22f523b75d9e3c4e90c476505',
        language: '',
      distractionFreeMode: true
    });
    gitalk.render('gitalk-container');
  }, window.Gitalk);
});
</script>

    </div>
</body>
</html>
